1 package org.apache.solr.common.util;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 import java.io.BufferedOutputStream;
21 import java.io.ByteArrayInputStream;
22 import java.io.ByteArrayOutputStream;
23 import java.io.File;
24 import java.io.FileOutputStream;
25 import java.io.IOException;
26 import java.io.InputStream;
27 import java.util.ArrayList;
28 import java.util.Arrays;
29 import java.util.Date;
30 import java.util.HashMap;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.Random;
34
35 import org.apache.commons.io.IOUtils;
36 import org.apache.lucene.util.TestUtil;
37 import org.apache.solr.SolrTestCaseJ4;
38 import org.apache.solr.common.EnumFieldValue;
39 import org.apache.solr.common.SolrDocument;
40 import org.apache.solr.common.SolrDocumentList;
41 import org.apache.solr.common.SolrInputDocument;
42 import org.apache.solr.common.SolrInputField;
43 import org.apache.solr.util.ConcurrentLRUCache;
44 import org.apache.solr.util.RTimer;
45 import org.junit.Test;
46 import org.noggit.CharArr;
47
48 public class TestJavaBinCodec extends SolrTestCaseJ4 {
49
50 private static final String SOLRJ_JAVABIN_BACKCOMPAT_BIN = "/solrj/javabin_backcompat.bin";
51 private final String BIN_FILE_LOCATION = "./solr/solrj/src/test-files/solrj/javabin_backcompat.bin";
52
53 private static final String SOLRJ_JAVABIN_BACKCOMPAT_BIN_CHILD_DOCS = "/solrj/javabin_backcompat_child_docs.bin";
54 private final String BIN_FILE_LOCATION_CHILD_DOCS = "./solr/solrj/src/test-files/solrj/javabin_backcompat_child_docs.bin";
55
56 public void testStrings() throws Exception {
57 JavaBinCodec javabin = new JavaBinCodec();
58 for (int i = 0; i < 10000 * RANDOM_MULTIPLIER; i++) {
59 String s = TestUtil.randomUnicodeString(random());
60 ByteArrayOutputStream os = new ByteArrayOutputStream();
61 javabin.marshal(s, os);
62 ByteArrayInputStream is = new ByteArrayInputStream(os.toByteArray());
63 Object o = javabin.unmarshal(is);
64 assertEquals(s, o);
65 }
66 }
67
68 private SolrDocument generateSolrDocumentWithChildDocs() {
69 SolrDocument parentDocument = new SolrDocument();
70 parentDocument.addField("id", "1");
71 parentDocument.addField("subject", "parentDocument");
72
73 SolrDocument childDocument = new SolrDocument();
74 childDocument.addField("id", "2");
75 childDocument.addField("cat", "foo");
76
77 SolrDocument secondKid = new SolrDocument();
78 secondKid.addField("id", "22");
79 secondKid.addField("cat", "bar");
80
81 SolrDocument grandChildDocument = new SolrDocument();
82 grandChildDocument.addField("id", "3");
83
84 childDocument.addChildDocument(grandChildDocument);
85 parentDocument.addChildDocument(childDocument);
86 parentDocument.addChildDocument(secondKid);
87
88 return parentDocument;
89 }
90
91 private List<Object> generateAllDataTypes() {
92 List<Object> types = new ArrayList<>();
93
94 types.add(null);
95 types.add(true);
96 types.add(false);
97 types.add((byte) 1);
98 types.add((short) 2);
99 types.add((double) 3);
100
101 types.add(-4);
102 types.add(4);
103 types.add(42);
104
105 types.add((long) -5);
106 types.add((long) 5);
107 types.add((long) 50);
108
109 types.add((float) 6);
110 types.add(new Date(0));
111
112 Map<Integer, Integer> map = new HashMap<>();
113 map.put(1, 2);
114 types.add(map);
115
116 SolrDocument doc = new SolrDocument();
117 doc.addField("foo", "bar");
118 types.add(doc);
119
120 SolrDocumentList solrDocs = new SolrDocumentList();
121 solrDocs.setMaxScore(1.0f);
122 solrDocs.setNumFound(1);
123 solrDocs.setStart(0);
124 solrDocs.add(0, doc);
125 types.add(solrDocs);
126
127 types.add(new byte[] {1,2,3,4,5});
128
129
130
131
132
133
134 types.add((byte) 15);
135
136 SolrInputDocument idoc = new SolrInputDocument();
137 idoc.addField("foo", "bar");
138 types.add(idoc);
139
140 SolrInputDocument parentDoc = new SolrInputDocument();
141 parentDoc.addField("foo", "bar");
142 SolrInputDocument childDoc = new SolrInputDocument();
143 childDoc.addField("foo", "bar");
144 parentDoc.addChildDocument(childDoc);
145 types.add(parentDoc);
146
147 types.add(new EnumFieldValue(1, "foo"));
148
149 types.add(map.entrySet().iterator().next());
150
151 types.add((byte) (1 << 5));
152
153 types.add("foo");
154 types.add(1);
155 types.add((long) 2);
156
157 SimpleOrderedMap simpleOrderedMap = new SimpleOrderedMap();
158 simpleOrderedMap.add("bar", "barbar");
159 types.add(simpleOrderedMap);
160
161 NamedList<String> nl = new NamedList<>();
162 nl.add("foo", "barbar");
163 types.add(nl);
164
165 return types;
166 }
167
168 @Test
169 public void testBackCompat() throws IOException {
170 JavaBinCodec javabin = new JavaBinCodec(){
171 @Override
172 public List<Object> readIterator(DataInputInputStream fis) throws IOException {
173 return super.readIterator(fis);
174 }
175 };
176 try {
177 InputStream is = getClass().getResourceAsStream(SOLRJ_JAVABIN_BACKCOMPAT_BIN);
178 List<Object> unmarshaledObj = (List<Object>) javabin.unmarshal(is);
179 List<Object> matchObj = generateAllDataTypes();
180
181 assertEquals(unmarshaledObj.size(), matchObj.size());
182 for(int i=0; i < unmarshaledObj.size(); i++) {
183
184 if(unmarshaledObj.get(i) instanceof byte[] && matchObj.get(i) instanceof byte[]) {
185 byte[] b1 = (byte[]) unmarshaledObj.get(i);
186 byte[] b2 = (byte[]) matchObj.get(i);
187 assertTrue(Arrays.equals(b1, b2));
188 } else if(unmarshaledObj.get(i) instanceof SolrDocument && matchObj.get(i) instanceof SolrDocument ) {
189 assertTrue(compareSolrDocument(unmarshaledObj.get(i), matchObj.get(i)));
190 } else if(unmarshaledObj.get(i) instanceof SolrDocumentList && matchObj.get(i) instanceof SolrDocumentList ) {
191 assertTrue(compareSolrDocumentList(unmarshaledObj.get(i), matchObj.get(i)));
192 } else if(unmarshaledObj.get(i) instanceof SolrInputDocument && matchObj.get(i) instanceof SolrInputDocument) {
193 assertTrue(compareSolrInputDocument(unmarshaledObj.get(i), matchObj.get(i)));
194 } else if(unmarshaledObj.get(i) instanceof SolrInputField && matchObj.get(i) instanceof SolrInputField) {
195 assertTrue(assertSolrInputFieldEquals(unmarshaledObj.get(i), matchObj.get(i)));
196 } else {
197 assertEquals(unmarshaledObj.get(i), matchObj.get(i));
198 }
199
200 }
201 } catch (IOException e) {
202 throw e;
203 }
204
205 }
206
207 @Test
208 public void testBackCompatForSolrDocumentWithChildDocs() throws IOException {
209 JavaBinCodec javabin = new JavaBinCodec(){
210 @Override
211 public List<Object> readIterator(DataInputInputStream fis) throws IOException {
212 return super.readIterator(fis);
213 }
214 };
215 try {
216 InputStream is = getClass().getResourceAsStream(SOLRJ_JAVABIN_BACKCOMPAT_BIN_CHILD_DOCS);
217 SolrDocument sdoc = (SolrDocument) javabin.unmarshal(is);
218 SolrDocument matchSolrDoc = generateSolrDocumentWithChildDocs();
219 assertTrue(compareSolrDocument(sdoc, matchSolrDoc));
220 } catch (IOException e) {
221 throw e;
222 }
223 }
224
225 @Test
226 public void testForwardCompat() throws IOException {
227 JavaBinCodec javabin = new JavaBinCodec();
228 ByteArrayOutputStream os = new ByteArrayOutputStream();
229
230 Object data = generateAllDataTypes();
231 try {
232 javabin.marshal(data, os);
233 byte[] newFormatBytes = os.toByteArray();
234
235 InputStream is = getClass().getResourceAsStream(SOLRJ_JAVABIN_BACKCOMPAT_BIN);
236 byte[] currentFormatBytes = IOUtils.toByteArray(is);
237
238 for (int i = 1; i < currentFormatBytes.length; i++) {
239 assertEquals(newFormatBytes[i], currentFormatBytes[i]);
240 }
241
242 } catch (IOException e) {
243 throw e;
244 }
245
246 }
247
248 @Test
249 public void testForwardCompatForSolrDocumentWithChildDocs() throws IOException {
250 JavaBinCodec javabin = new JavaBinCodec();
251 ByteArrayOutputStream os = new ByteArrayOutputStream();
252
253 SolrDocument sdoc = generateSolrDocumentWithChildDocs();
254 try {
255 javabin.marshal(sdoc, os);
256 byte[] newFormatBytes = os.toByteArray();
257
258 InputStream is = getClass().getResourceAsStream(SOLRJ_JAVABIN_BACKCOMPAT_BIN_CHILD_DOCS);
259 byte[] currentFormatBytes = IOUtils.toByteArray(is);
260
261 for (int i = 1; i < currentFormatBytes.length; i++) {
262 assertEquals(newFormatBytes[i], currentFormatBytes[i]);
263 }
264
265 } catch (IOException e) {
266 throw e;
267 }
268
269 }
270
271 @Test
272 public void testResponseChildDocuments() throws IOException {
273
274
275 JavaBinCodec javabin = new JavaBinCodec();
276 ByteArrayOutputStream baos = new ByteArrayOutputStream();
277 javabin.marshal(generateSolrDocumentWithChildDocs(), baos);
278
279 SolrDocument result = (SolrDocument) javabin.unmarshal(new ByteArrayInputStream(baos.toByteArray()));
280 assertEquals(2, result.size());
281 assertEquals("1", result.getFieldValue("id"));
282 assertEquals("parentDocument", result.getFieldValue("subject"));
283 assertTrue(result.hasChildDocuments());
284
285 List<SolrDocument> childDocuments = result.getChildDocuments();
286 assertNotNull(childDocuments);
287 assertEquals(2, childDocuments.size());
288 assertEquals(2, childDocuments.get(0).size());
289 assertEquals("2", childDocuments.get(0).getFieldValue("id"));
290 assertEquals("foo", childDocuments.get(0).getFieldValue("cat"));
291
292 assertEquals(2, childDocuments.get(1).size());
293 assertEquals("22", childDocuments.get(1).getFieldValue("id"));
294 assertEquals("bar", childDocuments.get(1).getFieldValue("cat"));
295 assertFalse(childDocuments.get(1).hasChildDocuments());
296 assertNull(childDocuments.get(1).getChildDocuments());
297
298 assertTrue(childDocuments.get(0).hasChildDocuments());
299 List<SolrDocument> grandChildDocuments = childDocuments.get(0).getChildDocuments();
300 assertNotNull(grandChildDocuments);
301 assertEquals(1, grandChildDocuments.size());
302 assertEquals(1, grandChildDocuments.get(0).size());
303 assertEquals("3", grandChildDocuments.get(0).getFieldValue("id"));
304 assertFalse(grandChildDocuments.get(0).hasChildDocuments());
305 assertNull(grandChildDocuments.get(0).getChildDocuments());
306 }
307 @Test
308 public void testStringCaching() throws Exception {
309 Map<String, Object> m = Utils.makeMap("key1", "val1", "key2", "val2");
310
311 ByteArrayOutputStream os1 = new ByteArrayOutputStream();
312 new JavaBinCodec().marshal(m, os1);
313 Map m1 = (Map) new JavaBinCodec().unmarshal(new ByteArrayInputStream(os1.toByteArray()));
314 ByteArrayOutputStream os2 = new ByteArrayOutputStream();
315 new JavaBinCodec().marshal(m, os2);
316 Map m2 = (Map) new JavaBinCodec().unmarshal(new ByteArrayInputStream(os2.toByteArray()));
317 List l1 = new ArrayList<>(m1.keySet());
318 List l2 = new ArrayList<>(m2.keySet());
319
320 assertTrue(l1.get(0).equals(l2.get(0)));
321 assertFalse(l1.get(0) == l2.get(0));
322 assertTrue(l1.get(1).equals(l2.get(1)));
323 assertFalse(l1.get(1) == l2.get(1));
324
325 JavaBinCodec.StringCache stringCache = new JavaBinCodec.StringCache(new Cache<JavaBinCodec.StringBytes, String>() {
326 private HashMap<JavaBinCodec.StringBytes, String> cache = new HashMap<>();
327
328 @Override
329 public String put(JavaBinCodec.StringBytes key, String val) {
330 return cache.put(key, val);
331 }
332
333 @Override
334 public String get(JavaBinCodec.StringBytes key) {
335 return cache.get(key);
336 }
337
338 @Override
339 public String remove(JavaBinCodec.StringBytes key) {
340 return cache.remove(key);
341 }
342
343 @Override
344 public void clear() {
345 cache.clear();
346
347 }
348 });
349
350
351 m1 = (Map) new JavaBinCodec(null, stringCache).unmarshal(new ByteArrayInputStream(os1.toByteArray()));
352 m2 = (Map) new JavaBinCodec(null, stringCache).unmarshal(new ByteArrayInputStream(os2.toByteArray()));
353 l1 = new ArrayList<>(m1.keySet());
354 l2 = new ArrayList<>(m2.keySet());
355 assertTrue(l1.get(0).equals(l2.get(0)));
356 assertTrue(l1.get(0) == l2.get(0));
357 assertTrue(l1.get(1).equals(l2.get(1)));
358 assertTrue(l1.get(1) == l2.get(1));
359
360
361 }
362
363 public void genBinaryFiles() throws IOException {
364 JavaBinCodec javabin = new JavaBinCodec();
365 ByteArrayOutputStream os = new ByteArrayOutputStream();
366
367 Object data = generateAllDataTypes();
368
369 javabin.marshal(data, os);
370 byte[] out = os.toByteArray();
371 FileOutputStream fs = new FileOutputStream(new File(BIN_FILE_LOCATION));
372 BufferedOutputStream bos = new BufferedOutputStream(fs);
373 bos.write(out);
374 bos.close();
375
376
377 javabin = new JavaBinCodec();
378 SolrDocument sdoc = generateSolrDocumentWithChildDocs();
379 os = new ByteArrayOutputStream();
380 javabin.marshal(sdoc, os);
381 fs = new FileOutputStream(new File(BIN_FILE_LOCATION_CHILD_DOCS));
382 bos = new BufferedOutputStream(fs);
383 bos.write(os.toByteArray());
384 bos.close();
385
386 }
387
388 private void testPerf() throws InterruptedException {
389 final ArrayList<JavaBinCodec.StringBytes> l = new ArrayList<>();
390 Cache<JavaBinCodec.StringBytes, String> cache = null;
391
392
393
394
395
396
397
398 Runtime.getRuntime().gc();
399 printMem("before cache init");
400
401 Cache<JavaBinCodec.StringBytes, String> cache1 = new Cache<JavaBinCodec.StringBytes, String>() {
402 private HashMap<JavaBinCodec.StringBytes, String> cache = new HashMap<>();
403
404 @Override
405 public String put(JavaBinCodec.StringBytes key, String val) {
406 l.add(key);
407 return cache.put(key, val);
408
409 }
410
411 @Override
412 public String get(JavaBinCodec.StringBytes key) {
413 return cache.get(key);
414 }
415
416 @Override
417 public String remove(JavaBinCodec.StringBytes key) {
418 return cache.remove(key);
419 }
420
421 @Override
422 public void clear() {
423 cache.clear();
424
425 }
426 };
427 final JavaBinCodec.StringCache STRING_CACHE = new JavaBinCodec.StringCache(cache1);
428
429
430 byte[] bytes = new byte[0];
431 JavaBinCodec.StringBytes stringBytes = new JavaBinCodec.StringBytes(null,0,0);
432
433 for(int i=0;i<10000;i++) {
434 String s = String.valueOf(random().nextLong());
435 int end = s.length();
436 int maxSize = end * 4;
437 if (bytes == null || bytes.length < maxSize) bytes = new byte[maxSize];
438 int sz = ByteUtils.UTF16toUTF8(s, 0, end, bytes, 0);
439 STRING_CACHE.get(stringBytes.reset(bytes, 0, sz));
440 }
441 printMem("after cache init");
442
443 RTimer timer = new RTimer();
444 final int ITERS = 1000000;
445 int THREADS = 10;
446
447 runInThreads(THREADS, new Runnable() {
448 @Override
449 public void run() {
450 JavaBinCodec.StringBytes stringBytes1 = new JavaBinCodec.StringBytes(new byte[0], 0, 0);
451 for (int i = 0; i < ITERS; i++) {
452 JavaBinCodec.StringBytes b = l.get(i % l.size());
453 stringBytes1.reset(b.bytes, 0, b.bytes.length);
454 if (STRING_CACHE.get(stringBytes1) == null) throw new RuntimeException("error");
455 }
456
457 }
458 });
459
460
461
462 printMem("after cache test");
463 System.out.println("time taken by LRUCACHE " + timer.getTime());
464 timer = new RTimer();
465
466 runInThreads(THREADS, new Runnable() {
467 @Override
468 public void run() {
469 String a = null;
470 CharArr arr = new CharArr();
471 for (int i = 0; i < ITERS; i++) {
472 JavaBinCodec.StringBytes sb = l.get(i % l.size());
473 arr.reset();
474 ByteUtils.UTF8toUTF16(sb.bytes, 0, sb.bytes.length, arr);
475 a = arr.toString();
476 }
477 }
478 });
479
480 printMem("after new string test");
481 System.out.println("time taken by string creation "+ timer.getTime());
482
483
484
485 }
486
487 private static void runInThreads(int count, Runnable runnable) throws InterruptedException {
488 ArrayList<Thread> t =new ArrayList();
489 for(int i=0;i<count;i++ ) t.add(new Thread(runnable));
490 for (Thread thread : t) thread.start();
491 for (Thread thread : t) thread.join();
492 }
493
494 static void printMem(String head) {
495 System.out.println("*************" + head + "***********");
496 int mb = 1024*1024;
497
498 Runtime runtime = Runtime.getRuntime();
499
500 System.out.println("Used Memory:"
501 + (runtime.totalMemory() - runtime.freeMemory()) / mb);
502
503
504 System.out.println("Free Memory:"
505 + runtime.freeMemory() / mb);
506
507
508 }
509
510 public static void main(String[] args) {
511
512
513 try {
514 doDecodePerf(args);
515 } catch (Exception e) {
516 throw new RuntimeException(e);
517 }
518 }
519
520
521 static String str(Random r, int sz) {
522 StringBuffer sb = new StringBuffer(sz);
523 for (int i=0; i<sz; i++) {
524 sb.append('\n' + r.nextInt(128-'\n'));
525 }
526 return sb.toString();
527 }
528
529
530 public static void doDecodePerf(String[] args) throws Exception {
531 int arg=0;
532 int nThreads = Integer.parseInt(args[arg++]);
533 int nBuffers = Integer.parseInt(args[arg++]);
534 final long iter = Long.parseLong(args[arg++]);
535 int cacheSz = Integer.parseInt(args[arg++]);
536
537 Random r = new Random(0);
538
539 final byte[][] buffers = new byte[nBuffers][];
540
541 for (int bufnum=0; bufnum<nBuffers; bufnum++) {
542 SolrDocument sdoc = new SolrDocument();
543 sdoc.put("id", "my_id_" + bufnum);
544 sdoc.put("author", str(r, 10 + r.nextInt(10)));
545 sdoc.put("address", str(r, 20 + r.nextInt(20)));
546 sdoc.put("license", str(r, 10));
547 sdoc.put("title", str(r, 5 + r.nextInt(10)));
548 sdoc.put("modified_dt", r.nextInt(1000000));
549 sdoc.put("creation_dt", r.nextInt(1000000));
550 sdoc.put("birthdate_dt", r.nextInt(1000000));
551 sdoc.put("clean", r.nextBoolean());
552 sdoc.put("dirty", r.nextBoolean());
553 sdoc.put("employed", r.nextBoolean());
554 sdoc.put("priority", r.nextInt(100));
555 sdoc.put("dependents", r.nextInt(6));
556 sdoc.put("level", r.nextInt(101));
557 sdoc.put("education_level", r.nextInt(10));
558
559 sdoc.put("state", "S"+r.nextInt(50));
560 sdoc.put("country", "Country"+r.nextInt(20));
561 sdoc.put("some_boolean", ""+r.nextBoolean());
562 sdoc.put("another_boolean", ""+r.nextBoolean());
563
564
565 JavaBinCodec javabin = new JavaBinCodec();
566 ByteArrayOutputStream os = new ByteArrayOutputStream();
567 javabin.marshal(sdoc, os);
568 os.toByteArray();
569 buffers[bufnum] = os.toByteArray();
570 }
571
572 int ret = 0;
573 final RTimer timer = new RTimer();
574 ConcurrentLRUCache underlyingCache = cacheSz > 0 ? new ConcurrentLRUCache<>(cacheSz,cacheSz-cacheSz/10,cacheSz,cacheSz/10,false,true,null) : null;
575 final JavaBinCodec.StringCache stringCache = underlyingCache==null ? null : new JavaBinCodec.StringCache(underlyingCache);
576 if (nThreads <= 0) {
577 ret += doDecode(buffers, iter, stringCache);
578 } else {
579 runInThreads(nThreads, new Runnable() {
580 @Override
581 public void run() {
582 try {
583 doDecode(buffers, iter, stringCache);
584 } catch (IOException e) {
585 e.printStackTrace();
586 }
587 }
588 });
589 }
590
591 long n = iter * Math.max(1,nThreads);
592 System.out.println("ret=" + ret + " THROUGHPUT=" + (n*1000 / timer.getTime()));
593 if (underlyingCache != null) System.out.println("cache: hits=" + underlyingCache.getStats().getCumulativeHits() + " lookups=" + underlyingCache.getStats().getCumulativeLookups() + " size=" + underlyingCache.getStats().getCurrentSize());
594 }
595
596 public static int doDecode(byte[][] buffers, long iter, JavaBinCodec.StringCache stringCache) throws IOException {
597 int ret = 0;
598 int bufnum = -1;
599
600 InputStream empty = new InputStream() {
601 @Override
602 public int read() throws IOException {
603 return -1;
604 }
605 };
606
607 while (--iter >= 0) {
608 if (++bufnum >= buffers.length) bufnum = 0;
609 byte[] buf = buffers[bufnum];
610 JavaBinCodec javabin = new JavaBinCodec(null, stringCache);
611 FastInputStream in = new FastInputStream(empty, buf, 0, buf.length);
612 Object o = javabin.unmarshal( in );
613 if (o instanceof SolrDocument) {
614 ret += ((SolrDocument) o).size();
615 }
616 }
617 return ret;
618 }
619
620 }
621
622